#!/usr/bin/python

# copyright Ginger Luo 2012(c)

################################################################
# TODO: 1. make this script system-independant
# TODO: 2. could be some gif/png files
# TODO: 3. central database, auto-detect if files are already downloaded
# TODO: 4. adding multi-thread
# TODO: 5. auto-foldering, together with #3
# FIXME: 6. in multi-thread, to avoid can't exit main thread, beacause exception happen in thread
# TODO: 7. in multi-thread, using bunch mode, e.g. 20 per thread once a time
#           thus no need to sync every pic num
# TODO: 8. Do we need pic tag? (add in jpg/gif/png field)
# FIXME: 9. multi-thread, can't exit normally (add timeout)
################################################################

#http://chan.sankakustatic.com/data/d7/80/d7809010e39f8ab43d9181cca8e72dba.png

#layer2:  http://idol.sankakucomplex.com/post/show/333591
#         http://chan.sankakucomplex.com/post/show/1743341

#SMPL2 = "http://idol.sankakucomplex.com/post/show/333591"

#URL = "http://idol.sankakucomplex.com/post/show/333591"

#<img alt="aqua_hair arms_up black_thighhighs blouse bow city cosplay detached_sleeves hairbow hatsune_miku karin_(1000giri) kneehighs kneesocks long_hair miniskirt pleated_skirt skirt sleeveless sleeveless_blouse sleeveless_shirt smile thighhighs tie twintails vocaloid" height="1000" id="image" onclick="Note.toggle();" orig_height="1280" orig_width="853" src="http://i1.sankakustatic.com/data/sample/50/90/preview509067ae9da298b24d8fd72fd533c29d.jpg" width="666">
PTN = r"http://.*?\.(?:jpg|gif|png)"
#PTN_GIF = r"http://.*?\.gif"

#http://img1.sankakustatic.com/tn/100/59859.jpg
#http://i1.sankakustatic.com/data/sample/50/90/preview509067ae9da298b24d8fd72fd533c29d.jpg
#http://idol.sankakustatic.com/data/50/90/509067ae9da298b24d8fd72fd533c29d.jpg

#http://chan.sankakustatic.com/data/72/d2/72d21515f5eb33f3df846b96e3e2631e.jpg
#http://c3.sankakustatic.com/data/sample/72/d2/preview72d21515f5eb33f3df846b96e3e2631e.jpg
PTN_PIC = r"http://.*?\.sankakustatic\.com/data/[/0-9a-f]+\.(?:jpg|gif|png)"

TIMEOUT = 100
ERROR_CNTR = 0
MAX_DIR_CAP = 500
NUM_PER_THREAD = 5
CHANNEL = 'chan'
import thread
tlock = thread.allocate_lock()
MAX_THREADS = 7

import urllib, socket
import re
import time
import os
import sys
import os.path as op
import copy
import random
from optparse import OptionParser
import anydbm

TM_START = time.time()

#os.system("mkdir -p pic_chan; mkdir -p pic_idol")
#try:
#  os.mkdir("pic_chan")
#  os.mkdir("pic_idol")
#except:
#  pass

# inner function
# get URL and tag
def get_pic_url(URL):
  try:
    uSMPL2 = urllib.urlopen(URL)
    lines = uSMPL2.readlines()
    uSMPL2.close()
  except:
    return ("", "")
  #print len(lines)

  PIC_PREVIEW = ""
  PIC_ORIGIN = ""
  tags = ''
  for l in lines:
    # if gif, get and quit
    #tmp = re.findall(PTN_GIF, l)
    #if tmp:
    #  PIC_ORIGIN = tmp[0]
    #  break
    tmp = re.findall(PTN, l)
    for t in tmp:
      prvtmp = re.findall(r'preview[0-9a-f]+\.(?:jpg|gif|png)', t)
      if prvtmp:
        PIC_PREVIEW = t
        continue
      orgtmp = re.findall(PTN_PIC, l)
      if orgtmp:
        PIC_ORIGIN = orgtmp[0]
        break
  
#  <title>aqua hair arms up black thighhighs blouse bow city cosplay detached sleeves hairbow hatsune miku karin (1000giri) kneehighs kneesocks long hair miniskirt pleated skirt skirt sleeveless sleeveless blouse sleeveless shirt smile thighhighs tie twintails vocaloid | Idol Complex - Idol & cosplay images</title>
  title = lines[4].lower().strip()  # the 4th line
  if "<title>" in title:
    tags = title[7:].split('|')[0].strip()

  #for l in lines:
  #  tmp = re.findall(r'preview[0-9a-f]+\.(?:jpg|gif|png)', l)
  #  if tmp:
  #    ttmp = re.findall('alt=".*?"', l)
  #    if ttmp:
  #      tags = ttmp[0][4].strip('"')
  #      break

  #print PIC_PREVIEW
  #print PIC_ORIGIN
  if PIC_ORIGIN:
    PIC_OUTPUT = PIC_ORIGIN
  else:
    PIC_OUTPUT = PIC_PREVIEW

  # will handle in upper
  #if not PIC_OUTPUT:
  #  print "Error with: ", URL
  #print PIC_OUTPUT
  return (PIC_OUTPUT, tags)


# save
def get_pic(sid, folder=''):
  #http://chan.sankakucomplex.com/post/show/1743341
  #CHANNEL = 'chan'
  URL = "http://%s.sankakucomplex.com/post/show/"%CHANNEL + sid
  print "[RETR]: ", URL
  po, tags = get_pic_url(URL)
  print "--> img src: ", po
  ext = op.splitext(po)
  try:
    if not po: raise IOError
    rd = urllib.urlopen(po).read()
    if folder:
      savepath = folder
    else:
      savepath = 'pic_' + CHANNEL + "_%05x"%(int(sid) / MAX_DIR_CAP)

    fn = op.join(savepath, CHANNEL + '_' + sid + ext[1])
    f = file(fn, 'wb')
    f.write(rd)
    f.close()
  except:
    print "Error with: ", URL
    #ERROR_CNTR += 1
    time.sleep(2)
  return tags

##  sync print
def sync_p(tid, s):
  # sync print
  tlock.acquire()
  print "[Thread %s]: %s"%(str(tid), s)
  tlock.release()

# atom threading
def thread_proc(t, stop):
  global pptr, exit_threads
  tid = t # thread ID
  sync_p(t, "THREAD created.")
  while True:
    # FIXME:
    # make this part a method,
    # and can be customized so as to accept file lists
    tlock.acquire()
    tmp = copy.copy(pptr)
    pptr += NUM_PER_THREAD
    tlock.release()
    # if it's the last?
    for x in range(tmp, tmp + NUM_PER_THREAD):
      if x >= stop:
        sync_p(t, "THREAD exited.")
        tlock.acquire()
        exit_threads += 1
        tlock.release()
        thread.exit_thread()
        break
      tags = get_pic(str(x))
      if tags:
        tdbs[t][str(x)] = tags

if __name__=='__main__':
  ################################################
  ##############    params fetch  ################

  parser = OptionParser() 
  parser.add_option('-c', '--channel', action='store', dest='channel', default='chan', help="specify channel, can be either 'idol' or 'chan', default is 'chan'")
  #parser.add_option("-p", "--pdbk", action="store_true", 
  #                  dest="pdcl", 
  #                  default=False, 
  #                  help="write pdbk data to oracle db")

  (options, args) = parser.parse_args()
  #print options
  #sys.exit(1)

  CHANNEL = options.channel
  print "channel: ", CHANNEL

  try:
    if len(args)==1:
      start = int(args[0])
      stop = start
    elif len(args)==2:
      start = int(args[0])
      stop = int(args[1])
      if start > stop:
        tmp = start
        start = stop
        stop = tmp
    else:
      raise AttributeError
  except:
    print 'wrong args, use -h.'
    sys,exit(0)

  print "START: %d, STOP: %d"%(start, stop)

  # create dirs
  if stop % MAX_DIR_CAP:
    rend = (stop / MAX_DIR_CAP) + 1
  else:
    rend = stop / MAX_DIR_CAP
  rstart = start / MAX_DIR_CAP
  #print rstart, rend
  for x in range(rstart, rend):
    tmp = "pic_" + CHANNEL + "_%05x"%x
    #print tmp
    try:
      os.mkdir(tmp)
    except OSError: # only bypass if dir exists
      pass

  #get_pic('1743341')
  #sys.exit(1)

  socket.setdefaulttimeout(TIMEOUT)
  # threading
  # thread var
  pptr = start
  exit_threads = 0
  # release threads
  tdbs = list()
  for x in range(MAX_THREADS):
    # tag database
    tdbs.append(anydbm.open("sankaku_%d.tdb"%x, 'c'))
    thread.start_new_thread(thread_proc, (x, stop))
  # main thread
  while True:
    #if pptr > stop: break
    tlock.acquire()
    local_et = copy.copy(exit_threads)
    if (pptr % 20)==0:
      os.system("echo %s %d > CURRENT"%(CHANNEL, pptr))
    tlock.release()
    if local_et==MAX_THREADS:
      break
    #print local_et
    time.sleep(10)

  sync_p('main', "THREAD exited.")

  # summary
  time_used = time.time() - TM_START
  print "%d secs elapsed."%(time_used)
  print "pics: " + str(stop - start)
  print "milisecs per PIC: ", str(1000.0 * time_used / (stop - start + 1))
  print "threads: " + str(MAX_THREADS)

  for x in range(MAX_THREADS):
    tdbs[x].close()

  # performance recording


  # revision 9

  # performance data:
  #3301 secs elapsed.
  #pics: 4000
  #milisecs per PIC:  825.150212417
  #thread 5
  #
  #
  #60 secs elapsed.
  #pics: 101
  #milisecs per PIC:  595.297029703
  #threads: 10
  #
  #110 secs elapsed.
  #pics: 31
  #milisecs per PIC:  3550.09677333
  #threads: 1
  #

  #530 secs elapsed.
  #pics: 871
  #milisecs per PIC:  609.071182466
  #threads: 7
else:
  pass
